package com.terren.spider.core.html.processor;

import java.util.HashSet;
import java.util.Set;

import com.terren.spider.core.html.configuration.Configuration;
import com.terren.spider.entity.config.SysConfig;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.processor.PageProcessor;

public class HtmlPageProcessor extends BasicHtmlProcessor implements PageProcessor {

	public HtmlPageProcessor(Configuration configuration) throws Exception {
		super(configuration);
	}


	@Override
	public void process(Page page) {
		logMessage(page);
		/***
		 * 分析目标页
		 */
		if (this.isTargetUrl(page, configuration.getEntry())) {// 如果页面是 目标页
			try {
				/***
				 * 加入存储队列
				 */
				Page collectItem = page;
				page.putField(SysConfig.COLLECT_ITEM_KEY, collectItem);// 把page对象放入pipeline 等待分析
				page.putField(SysConfig.COLLECT_TOTAL_TARGET_URL, getTargetUrlCount());// 把page对象放入pipeline 等待分析
				Set<String> paginationUrl = new HashSet<>();
				paginationUrl.addAll(this.findTargetPaginationUrl(page,  configuration));// 分页
				this.addTargetRequests(page,paginationUrl);
			} catch (Exception e) {
				log.error("process parse error", e);
			}
		}
	}
}
